package com.wk.crawler;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import java.io.IOException;
import java.security.cert.X509Certificate;

public class BlogCrawler {

    public static void main(String[] args) throws Exception{


        // 设置忽略SSL验证的信任管理器
        TrustManager[] trustAll = new TrustManager[]{
                new X509TrustManager() {
                    @Override
                    public X509Certificate[] getAcceptedIssuers() {
                        return null;
                    }

                    @Override
                    public void checkClientTrusted(X509Certificate[] certs, String authType) {
                    }

                    @Override
                    public void checkServerTrusted(X509Certificate[] certs, String authType) {
                    }
                }
        };

        SSLContext sslContext = SSLContext.getInstance("TLS");
        sslContext.init(null, trustAll, new java.security.SecureRandom());
        HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory());

        // 禁用主机名验证
        HttpsURLConnection.setDefaultHostnameVerifier((hostname, session) -> true);

        String url = "https://www.macw.cc/talks"; // 替换为目标专栏的URL
        try {
            // 发送HTTP请求获取网页内容
            Document doc = Jsoup.connect(url).get();

            // 解析网页内容，提取相关信息
            Elements articles = doc.select("div.article"); // 使用合适的CSS选择器定位文章列表

            // 遍历提取的文章列表
            for (Element article : articles) {
                String title = article.select("h2.title").text(); // 文章标题
                String content = article.select("div.content").text(); // 文章内容
                System.out.println("Title: " + title);
                System.out.println("Content: " + content);
                System.out.println("--------------");
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}
